{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Wide and Deep模型训练和测试  \n",
    "1. 生成WD训练数据    \n",
    "2. 生成WD测试数据  \n",
    "3. 训练Wide and Deep model  \n",
    "4. WD模型评估和预测"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. 生成WD训练数据  \n",
    "注意label和columns的顺序，送入input_fn需要舍去head"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def trainDataForWD(input_file,output_file):\n",
    "    dpath = \"./data/\"\n",
    "    fin = open(dpath+input_file,\"r\")\n",
    "    fout = open(dpath+output_file,\"w\")\n",
    "    start = 0\n",
    "    for line in fin:\n",
    "        if start==0:\n",
    "            start+=1\n",
    "            continue\n",
    "        cols = line.strip().split(\",\")\n",
    "        outcols = [cols[-1]]+cols[-6:-1]+[cols[-10]]+cols[-9:-6]+cols[2:-10]\n",
    "        fout.write(\",\".join(outcols)+\"\\n\")\n",
    "    fin.close()\n",
    "    fout.close()       "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 28.5 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "trainDataForWD(\"train_final.csv\",\"train_wd.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_wd = pd.read_csv(\"./data/train_wd.csv\",header=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>14</th>\n",
       "      <th>15</th>\n",
       "      <th>16</th>\n",
       "      <th>17</th>\n",
       "      <th>18</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>15.581</td>\n",
       "      <td>0.795</td>\n",
       "      <td>0.506</td>\n",
       "      <td>0.474</td>\n",
       "      <td>0.53734</td>\n",
       "      <td>-0.25183</td>\n",
       "      <td>17</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1.991</td>\n",
       "      <td>-0.081</td>\n",
       "      <td>0.743</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.99810</td>\n",
       "      <td>0.23361</td>\n",
       "      <td>6</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>3</td>\n",
       "      <td>13</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>7</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>1.991</td>\n",
       "      <td>-0.072</td>\n",
       "      <td>0.743</td>\n",
       "      <td>0.500</td>\n",
       "      <td>-0.04898</td>\n",
       "      <td>-0.13422</td>\n",
       "      <td>6</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>3</td>\n",
       "      <td>13</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>7</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>1.991</td>\n",
       "      <td>-0.081</td>\n",
       "      <td>0.743</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.99810</td>\n",
       "      <td>0.05294</td>\n",
       "      <td>40</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>3</td>\n",
       "      <td>13</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>7</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>15.581</td>\n",
       "      <td>1.211</td>\n",
       "      <td>0.506</td>\n",
       "      <td>0.364</td>\n",
       "      <td>0.38750</td>\n",
       "      <td>-0.36784</td>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   0       1      2      3      4        5        6   7   8   9   10  11  12  \\\n",
       "0   1  15.581  0.795  0.506  0.474  0.53734 -0.25183  17   8   0   1   6   5   \n",
       "1   1   1.991 -0.081  0.743  1.000  0.99810  0.23361   6   8   0   3   7   3   \n",
       "2   1   1.991 -0.072  0.743  0.500 -0.04898 -0.13422   6   8   0   3   7   3   \n",
       "3   1   1.991 -0.081  0.743  1.000  0.99810  0.05294  40   0   0   3   7   3   \n",
       "4   1  15.581  1.211  0.506  0.364  0.38750 -0.36784   1   8   0   1   6   5   \n",
       "\n",
       "   13  14  15  16  17  18  \n",
       "0   1   0   2   7   8  13  \n",
       "1  13   4   0   9   7  13  \n",
       "2  13   4   0   9   7  13  \n",
       "3  13   4   0   9   7  13  \n",
       "4   1   0   2   7   8  13  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_wd.iloc[0:5,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7377403, 19)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_wd.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. 生成WD测试数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def testDataForWD(input_file,output_file):\n",
    "    dpath = \"./data/\"\n",
    "    fin = open(dpath+input_file,\"r\")\n",
    "    fout = open(dpath+output_file,\"w\")\n",
    "    start = 0\n",
    "    for line in fin:\n",
    "        if start==0:\n",
    "            start+=1\n",
    "            continue\n",
    "        cols = line.strip().split(\",\")\n",
    "        outcols = cols[-5:]+[cols[-9]]+cols[-8:-5]+cols[3:-9]\n",
    "        fout.write(\",\".join(outcols)+\"\\n\")\n",
    "    fin.close()\n",
    "    fout.close()       "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 9.62 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "testDataForWD(\"test_final.csv\",\"test_wd.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_wd = pd.read_csv(\"./data/test_wd.csv\",header=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>14</th>\n",
       "      <th>15</th>\n",
       "      <th>16</th>\n",
       "      <th>17</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.401</td>\n",
       "      <td>2.944</td>\n",
       "      <td>0.366</td>\n",
       "      <td>0.504</td>\n",
       "      <td>0.38083</td>\n",
       "      <td>-0.14208</td>\n",
       "      <td>24</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>12</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.401</td>\n",
       "      <td>32.878</td>\n",
       "      <td>0.366</td>\n",
       "      <td>0.625</td>\n",
       "      <td>0.53185</td>\n",
       "      <td>0.45662</td>\n",
       "      <td>25</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>12</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.594</td>\n",
       "      <td>-0.072</td>\n",
       "      <td>0.144</td>\n",
       "      <td>0.400</td>\n",
       "      <td>-0.88885</td>\n",
       "      <td>0.42822</td>\n",
       "      <td>12</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>16</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>12</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.034</td>\n",
       "      <td>-0.029</td>\n",
       "      <td>0.296</td>\n",
       "      <td>0.226</td>\n",
       "      <td>-0.18739</td>\n",
       "      <td>0.23750</td>\n",
       "      <td>25</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>11</td>\n",
       "      <td>7</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "      <td>3</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-0.034</td>\n",
       "      <td>-0.072</td>\n",
       "      <td>0.296</td>\n",
       "      <td>0.400</td>\n",
       "      <td>0.98938</td>\n",
       "      <td>-0.30702</td>\n",
       "      <td>32</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>11</td>\n",
       "      <td>7</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "      <td>3</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      0       1      2      3        4        5   6   7   8   9   10  11  12  \\\n",
       "0 -0.401   2.944  0.366  0.504  0.38083 -0.14208  24   4   0   3   7   2   1   \n",
       "1 -0.401  32.878  0.366  0.625  0.53185  0.45662  25   4   0   3   7   2   1   \n",
       "2 -0.594  -0.072  0.144  0.400 -0.88885  0.42822  12   2   0   0  16   9   1   \n",
       "3 -0.034  -0.029  0.296  0.226 -0.18739  0.23750  25   8   0   7  11   7   3   \n",
       "4 -0.034  -0.072  0.296  0.400  0.98938 -0.30702  32   0   0   7  11   7   3   \n",
       "\n",
       "   13  14  15  16  17  \n",
       "0   0   2   7  12  13  \n",
       "1   0   2   7  12  13  \n",
       "2   0   2   4  12  12  \n",
       "3   5   1   9   3  13  \n",
       "4   5   1   9   3  13  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_wd.iloc[0:5,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2556790, 18)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_wd.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. 训练Wide and Deep model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using TensorFlow version 1.13.1\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import time\n",
    "import tensorflow as tf\n",
    "\n",
    "tf.logging.set_verbosity(tf.logging.INFO) # Set to INFO for tracking training, default is WARN. ERROR for least messages\n",
    "\n",
    "print(\"Using TensorFlow version %s\\n\" % (tf.__version__))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1.构建feature columns "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using TensorFlow version 1.13.1\n",
      "\n",
      "Feature columns are:  ['user_pop', 'item_pop', 'user_rate', 'item_rate', 'lfm_reco', 'song_length', 'genre_ids', 'language', 'mult_genre', 'source_system_tab', 'source_screen_name', 'source_type', 'city', 'bd', 'gender', 'registered_via', 'registration_init_time', 'expiration_date'] \n",
      "\n"
     ]
    }
   ],
   "source": [
    "CONTINUOUS_COLUMNS =  [\"user_pop\",\"item_pop\",\"user_rate\",\"item_rate\",\"lfm_reco\",\"song_length\"] \n",
    "CATEGORICAL_COLUMNS = [\"genre_ids\",\"language\",\"mult_genre\",\"source_system_tab\",\"source_screen_name\",\"source_type\",\\\n",
    "                       \"city\",\"bd\",\"gender\",\"registered_via\",\"registration_init_time\",\"expiration_date\"] \n",
    "LABEL_COLUMN = [\"target\"]\n",
    "\n",
    "TRAIN_DATA_COLUMNS = LABEL_COLUMN+ CONTINUOUS_COLUMNS + CATEGORICAL_COLUMNS\n",
    "TEST_DATA_COLUMNS = CONTINUOUS_COLUMNS + CATEGORICAL_COLUMNS\n",
    "\n",
    "FEATURE_COLUMNS = CONTINUOUS_COLUMNS + CATEGORICAL_COLUMNS\n",
    "\n",
    "print('Feature columns are: ', FEATURE_COLUMNS, '\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2.读取数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input function configured\n"
     ]
    }
   ],
   "source": [
    "def generate_input_fn(filepath, batch_size):\n",
    "    def _input_fn():\n",
    "        filepath_queue = tf.train.string_input_producer([filepath])\n",
    "        reader = tf.TextLineReader()\n",
    "        key, value = reader.read_up_to(filepath_queue, num_records=batch_size)\n",
    "\n",
    "        cont_defaults = [ [0.] for i in range(6) ]\n",
    "        cate_defaults = [ [\" \"] for i in range(12) ]\n",
    "        label_defaults = [ [0] ]\n",
    "        \n",
    "        if filepath==\"./data/train_wd.csv\":\n",
    "            column_headers = TRAIN_DATA_COLUMNS\n",
    "            record_defaults = label_defaults + cont_defaults + cate_defaults\n",
    "            columns = tf.decode_csv(\n",
    "                value, record_defaults=record_defaults)\n",
    "            all_columns = dict(zip(column_headers, columns))\n",
    "            labels = all_columns.pop(LABEL_COLUMN[0])\n",
    "            features = all_columns \n",
    "            for feature_name in CATEGORICAL_COLUMNS:\n",
    "                features[feature_name] = tf.expand_dims(features[feature_name], -1)\n",
    "            return features, labels\n",
    "        \n",
    "        if filepath==\"./data/test_wd.csv\":\n",
    "            column_headers = TEST_DATA_COLUMNS\n",
    "            record_defaults = cont_defaults + cate_defaults\n",
    "            columns = tf.decode_csv(\n",
    "                value, record_defaults=record_defaults)\n",
    "            all_columns = dict(zip(column_headers, columns))\n",
    "            features = all_columns \n",
    "            for feature_name in CATEGORICAL_COLUMNS:\n",
    "                features[feature_name] = tf.expand_dims(features[feature_name], -1)\n",
    "            return features\n",
    "            \n",
    "    return _input_fn\n",
    "\n",
    "print('input function configured')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "3.构建特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wide/Sparse columns configured\n"
     ]
    }
   ],
   "source": [
    "wide_columns = []\n",
    "for name in CATEGORICAL_COLUMNS:\n",
    "    wide_columns.append(tf.feature_column.categorical_column_with_hash_bucket(\n",
    "            name, hash_bucket_size=512))\n",
    "\n",
    "print('Wide/Sparse columns configured')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "deep/continuous columns configured\n"
     ]
    }
   ],
   "source": [
    "deep_columns = []\n",
    "for name in CONTINUOUS_COLUMNS:\n",
    "    deep_columns.append(tf.feature_column.numeric_column(name))\n",
    "\n",
    "print('deep/continuous columns configured')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "wide and deep columns configured\n"
     ]
    }
   ],
   "source": [
    "# Embeddings for wide columns into deep columns\n",
    "for col in wide_columns:\n",
    "    deep_columns.append(tf.feature_column.embedding_column\\\n",
    "                        (col, dimension=9))\n",
    "\n",
    "print('wide and deep columns configured')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "4.初始化WD模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_model_dir(model_type):\n",
    "    return 'WD_models/model_' + model_type + '_' + str(int(time.time()))\n",
    "\n",
    "def get_model(model_type, model_dir):\n",
    "    print(\"Model directory = %s\" % model_dir)\n",
    "\n",
    "    runconfig = tf.estimator.RunConfig(\n",
    "        save_checkpoints_secs=None,\n",
    "        save_checkpoints_steps = 100,)\n",
    "    \n",
    "    model = None\n",
    "    \n",
    "    # Linear Classifier\n",
    "    if model_type == 'WIDE':\n",
    "        model = tf.estimator.LinearClassifier(\n",
    "            model_dir=model_dir, \n",
    "            feature_columns=wide_columns,\n",
    "            optimizer=tf.train.FtrlOptimizer(0.1, l2_regularization_strength=1.0))\n",
    "\n",
    "    # Deep Neural Net Classifier\n",
    "    if model_type == 'DEEP':\n",
    "        model = tf.estimator.DNNClassifier(\n",
    "            model_dir=model_dir,\n",
    "            feature_columns=deep_columns,\n",
    "            hidden_units=[64,16],\n",
    "            optimizer= tf.train.ProximalAdagradOptimizer(learning_rate=0.1, \\\n",
    "                            l1_regularization_strength=0.001,l2_regularization_strength=0.001),)\n",
    "\n",
    "    # Combined Linear and Deep Classifier\n",
    "    if model_type == 'WIDE_AND_DEEP':\n",
    "        model = tf.estimator.DNNLinearCombinedClassifier(\n",
    "            model_dir=model_dir,\n",
    "            linear_feature_columns=wide_columns,\n",
    "            linear_optimizer=tf.train.FtrlOptimizer(0.1, l2_regularization_strength=1.0),\n",
    "            dnn_feature_columns=deep_columns,\n",
    "            dnn_optimizer= tf.train.ProximalAdagradOptimizer(learning_rate=0.1, \\\n",
    "                            l1_regularization_strength=0.001,l2_regularization_strength=0.001),\n",
    "            dnn_hidden_units=[128,64,32,16],\n",
    "            config=runconfig)\n",
    "        \n",
    "    print('estimator built')\n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "5.训练WD模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_file = \"./data/train_wd.csv\"\n",
    "test_file  = \"./data/test_wd.csv\"\n",
    "train_sample_size = 7377403\n",
    "BATCH_SIZE = 2000\n",
    "train_steps = train_sample_size/BATCH_SIZE # 7377403/2000 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model directory = WD_models/model_WIDE_AND_DEEP_1567750302\n",
      "INFO:tensorflow:Using config: {'_model_dir': 'WD_models/model_WIDE_AND_DEEP_1567750302', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': 100, '_save_checkpoints_secs': None, '_session_config': allow_soft_placement: true\n",
      "graph_options {\n",
      "  rewrite_options {\n",
      "    meta_optimizer_iterations: ONE\n",
      "  }\n",
      "}\n",
      ", '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x0000021803D17048>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1}\n",
      "estimator built\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "WARNING:tensorflow:From D:\\Program Files\\Anaconda\\lib\\site-packages\\tensorflow\\python\\feature_column\\feature_column_v2.py:2997: HashedCategoricalColumn._num_buckets (from tensorflow.python.feature_column.feature_column_v2) is deprecated and will be removed after 2018-11-30.\n",
      "Instructions for updating:\n",
      "The old _FeatureColumn APIs are being deprecated. Please use the new FeatureColumn APIs instead.\n",
      "WARNING:tensorflow:From D:\\Program Files\\Anaconda\\lib\\site-packages\\tensorflow\\python\\ops\\array_grad.py:425: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "WARNING:tensorflow:From D:\\Program Files\\Anaconda\\lib\\site-packages\\tensorflow\\python\\training\\monitored_session.py:809: start_queue_runners (from tensorflow.python.training.queue_runner_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "To construct input pipelines, use the `tf.data` module.\n",
      "INFO:tensorflow:Saving checkpoints for 0 into WD_models/model_WIDE_AND_DEEP_1567750302\\model.ckpt.\n",
      "INFO:tensorflow:loss = 1723.3744, step = 1\n",
      "INFO:tensorflow:Saving checkpoints for 100 into WD_models/model_WIDE_AND_DEEP_1567750302\\model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 18.5789\n",
      "INFO:tensorflow:loss = 898.37, step = 101 (5.396 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 200 into WD_models/model_WIDE_AND_DEEP_1567750302\\model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 21.4391\n",
      "INFO:tensorflow:loss = 894.0593, step = 201 (4.654 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 300 into WD_models/model_WIDE_AND_DEEP_1567750302\\model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 21.0096\n",
      "INFO:tensorflow:loss = 991.00165, step = 301 (4.769 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 400 into WD_models/model_WIDE_AND_DEEP_1567750302\\model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 22.7395\n",
      "INFO:tensorflow:loss = 937.29175, step = 401 (4.395 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 500 into WD_models/model_WIDE_AND_DEEP_1567750302\\model.ckpt.\n",
      "WARNING:tensorflow:From D:\\Program Files\\Anaconda\\lib\\site-packages\\tensorflow\\python\\training\\saver.py:966: remove_checkpoint (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use standard file APIs to delete files with this prefix.\n",
      "INFO:tensorflow:global_step/sec: 20.6974\n",
      "INFO:tensorflow:loss = 1030.0807, step = 501 (4.828 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 600 into WD_models/model_WIDE_AND_DEEP_1567750302\\model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 22.8179\n",
      "INFO:tensorflow:loss = 1000.09436, step = 601 (4.389 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 700 into WD_models/model_WIDE_AND_DEEP_1567750302\\model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 21.5308\n",
      "INFO:tensorflow:loss = 1025.4204, step = 701 (4.632 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 800 into WD_models/model_WIDE_AND_DEEP_1567750302\\model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 20.9409\n",
      "INFO:tensorflow:loss = 1050.8049, step = 801 (4.775 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 900 into WD_models/model_WIDE_AND_DEEP_1567750302\\model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 22.5684\n",
      "INFO:tensorflow:loss = 1057.6697, step = 901 (4.441 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 1000 into WD_models/model_WIDE_AND_DEEP_1567750302\\model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 22.1597\n",
      "INFO:tensorflow:loss = 1061.0381, step = 1001 (4.515 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 1100 into WD_models/model_WIDE_AND_DEEP_1567750302\\model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 22.7201\n",
      "INFO:tensorflow:loss = 1100.4081, step = 1101 (4.394 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 1200 into WD_models/model_WIDE_AND_DEEP_1567750302\\model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 22.6586\n",
      "INFO:tensorflow:loss = 1135.7129, step = 1201 (4.422 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 1300 into WD_models/model_WIDE_AND_DEEP_1567750302\\model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 21.7512\n",
      "INFO:tensorflow:loss = 1185.9873, step = 1301 (4.585 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 1400 into WD_models/model_WIDE_AND_DEEP_1567750302\\model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 21.9012\n",
      "INFO:tensorflow:loss = 1105.0598, step = 1401 (4.577 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 1500 into WD_models/model_WIDE_AND_DEEP_1567750302\\model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 22.2253\n",
      "INFO:tensorflow:loss = 1157.8938, step = 1501 (4.500 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 1600 into WD_models/model_WIDE_AND_DEEP_1567750302\\model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 21.8134\n",
      "INFO:tensorflow:loss = 1096.2549, step = 1601 (4.587 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 1700 into WD_models/model_WIDE_AND_DEEP_1567750302\\model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 21.9661\n",
      "INFO:tensorflow:loss = 1166.0789, step = 1701 (4.551 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 1800 into WD_models/model_WIDE_AND_DEEP_1567750302\\model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 22.3293\n",
      "INFO:tensorflow:loss = 1131.9438, step = 1801 (4.466 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 1900 into WD_models/model_WIDE_AND_DEEP_1567750302\\model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 21.3051\n",
      "INFO:tensorflow:loss = 1181.2661, step = 1901 (4.703 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 2000 into WD_models/model_WIDE_AND_DEEP_1567750302\\model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 22.4631\n",
      "INFO:tensorflow:loss = 1053.9304, step = 2001 (4.455 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 2100 into WD_models/model_WIDE_AND_DEEP_1567750302\\model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 22.2314\n",
      "INFO:tensorflow:loss = 1107.2708, step = 2101 (4.494 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 2200 into WD_models/model_WIDE_AND_DEEP_1567750302\\model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 20.0293\n",
      "INFO:tensorflow:loss = 1134.6902, step = 2201 (4.993 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 2300 into WD_models/model_WIDE_AND_DEEP_1567750302\\model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 21.5395\n",
      "INFO:tensorflow:loss = 1179.6315, step = 2301 (4.645 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 2400 into WD_models/model_WIDE_AND_DEEP_1567750302\\model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 18.0883\n",
      "INFO:tensorflow:loss = 1198.7268, step = 2401 (5.528 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 2500 into WD_models/model_WIDE_AND_DEEP_1567750302\\model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 20.8626\n",
      "INFO:tensorflow:loss = 1149.2905, step = 2501 (4.790 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 2600 into WD_models/model_WIDE_AND_DEEP_1567750302\\model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 20.9636\n",
      "INFO:tensorflow:loss = 1109.4211, step = 2601 (4.773 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 2700 into WD_models/model_WIDE_AND_DEEP_1567750302\\model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 20.3187\n",
      "INFO:tensorflow:loss = 1121.3032, step = 2701 (4.916 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 2800 into WD_models/model_WIDE_AND_DEEP_1567750302\\model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 22.08\n",
      "INFO:tensorflow:loss = 1145.6583, step = 2801 (4.524 sec)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Saving checkpoints for 2900 into WD_models/model_WIDE_AND_DEEP_1567750302\\model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 22.1678\n",
      "INFO:tensorflow:loss = 1088.8809, step = 2901 (4.523 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 3000 into WD_models/model_WIDE_AND_DEEP_1567750302\\model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 20.1648\n",
      "INFO:tensorflow:loss = 1224.4978, step = 3001 (4.960 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 3100 into WD_models/model_WIDE_AND_DEEP_1567750302\\model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 21.7548\n",
      "INFO:tensorflow:loss = 1110.1195, step = 3101 (4.585 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 3200 into WD_models/model_WIDE_AND_DEEP_1567750302\\model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 22.0338\n",
      "INFO:tensorflow:loss = 1046.283, step = 3201 (4.544 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 3300 into WD_models/model_WIDE_AND_DEEP_1567750302\\model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 21.1546\n",
      "INFO:tensorflow:loss = 1237.0164, step = 3301 (4.734 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 3400 into WD_models/model_WIDE_AND_DEEP_1567750302\\model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 22.0747\n",
      "INFO:tensorflow:loss = 1115.7673, step = 3401 (4.528 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 3500 into WD_models/model_WIDE_AND_DEEP_1567750302\\model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 21.6876\n",
      "INFO:tensorflow:loss = 1083.845, step = 3501 (4.613 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 3600 into WD_models/model_WIDE_AND_DEEP_1567750302\\model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 20.4869\n",
      "INFO:tensorflow:loss = 1171.7599, step = 3601 (4.872 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 3689 into WD_models/model_WIDE_AND_DEEP_1567750302\\model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 788.49615.\n",
      "wide and deep model fit done\n",
      "Wall time: 3min 11s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "MODEL_TYPE = 'WIDE_AND_DEEP'\n",
    "model_dir = create_model_dir(model_type=MODEL_TYPE)\n",
    "wd_model = get_model(model_type=MODEL_TYPE, model_dir=model_dir)\n",
    "wd_model.train(input_fn=generate_input_fn(train_file, BATCH_SIZE), steps=train_steps)\n",
    "print('wide and deep model fit done')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4. WD模型评估和预测"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "#predict返回一个生成器\n",
    "train_results = wd_model.predict(input_fn=generate_input_fn(train_file, BATCH_SIZE))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_results = wd_model.predict(input_fn=generate_input_fn(test_file, BATCH_SIZE))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1.评估"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import metrics\n",
    "from sklearn.metrics import roc_auc_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from WD_models/model_WIDE_AND_DEEP_1567750302\\model.ckpt-3689\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    }
   ],
   "source": [
    "train_proba = []\n",
    "for i in range(7377403):\n",
    "    one_res = next(train_results)[\"probabilities\"]\n",
    "    train_proba.append(one_res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_proba = np.array(train_proba)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7377403, 2)"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_proba.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_target = train_wd.iloc[:,0].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7377403,)"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_target.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7763867628090612"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "roc_auc_score(train_target,train_proba[:,1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "fpr,tpr,thresholds = metrics.roc_curve(train_target,train_proba[:,1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7763867628090612"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics.auc(fpr, tpr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl4VeW59/HvTUiABAhkYAqEMARlEEGjCNaqRRRsKz1WW+2kdrCttZ5aT1s97WWtvj3Wenztaes5lvb42lonHKpYcah11soo82RkyAQkEBICmZP7/WNv0xgTCCErK8n+fa6Li7X2frL3/UCyf3meNTzm7oiIiAD0CbsAERHpPhQKIiLSRKEgIiJNFAoiItJEoSAiIk0UCiIi0kShICIiTRQKIsfIzF41s6+HXYdIEBQKIiLSRKEg0gXMrG/YNYi0h0JBYpaZXWVmzzTbzzWzxc32881shpnNM7MtZlZuZr8FrB2vfaWZvWVmd5tZKXCLmfUxs5+Y2S4zKzazP5lZcrOv+ZiZvW1mZdH3vrKTuyxyVAoFiWWvAWdFP6xHAvHAmQBmNh4YCBQBTwA/AdKA9z9o0w6zgO3AMODnwJXRP+cCH7z+b6Pvlwk8B/wGSAdmAGuOs38ix0yhIDHL3bcDFUQ+gM8GXgAKzezE6P4bwHxgk7s/7u51wK+APe18iyJ3/42717t7FfBF4P+6+3Z3PwTcBFwWnVr6IvCSuz/s7nXuvt/dFQrS5TTPKbHuNeAcYGJ0u4xIIMyO7o8C8j9o7O5uZvkffZlWtWw3CtjVbH8XkZ/B4cAYIqMQkVBppCCx7oNQOCu6/RqRUDg7ur2byAc2AGZmzfePouV96YuAsc32M4F6YC+RAJlwzNWLdDKFgsS614jM8Q9w9wL+OWWUCrwLPAtMNbOLo9M81wEjOvheDwPXm9k4MxsI/AfwqLvXAw8C55nZ58ysr5mlmtmM4+uayLFTKEhMc/dtwCEiYYC7HyRycPgtd29w933ApcAvgP1ANvBWB9/uPuAB4HVgB1ANfDf6vnnAhcANQCmRg8wnd/B9RDrMtPKaiIh8QCMFERFpolAQ6SAzu9fMDrXy596waxPpKE0fiYhIkx53nUJaWppnZWWFXYaISI+yatWqfe6efrR2PS4UsrKyWLlyZdhliIj0KGa26+itdExBRESaUSiIiEgThYKIiDRRKIiISBOFgoiINAksFMzsvujqUhvaeN7M7NfR1a7WmdkpQdUiIiLtE+RI4X4id5tsywIiNxfLBq4G/ifAWkREpB0Cu07B3V83s6wjNFkI/Mkjl1S/Y2ZDzGyku+8OqiYRkc5Q19DIwao6DtXUU1FdT2VtA5W19VTVNlBd30BtfSO1DU5dfSO1DY00utP85hEf3Enig8f8Q9sffe6DnbmTh3PymCGB9i3Mi9cy+PDKVAXRxz4SCmZ2NZHRBJmZmV1SnIjEjsraeooP1lByqIaSihr2Haph36Fa9h+qofRwLQcqaymrrKOsso7yqjqq6hq6vEYzGDa4f68OBWvlsVZvxOTui4BFADk5ObpZk4i0W0V1HUVl1RSVV7G7rJrd5VXsLq9mT3k1ew5G/j5UU9/q1w5NjCclKYGUpAQyUxI5KSOe5AHxDB4Qz+D+fRnUP56kfn0Z2K8vif3iGBAfR//4OBL69iE+zugXF0d8X6OPRT7uon9hWLPt6N9mzbb/+VhXCzMUCvjwsoajiSxXKCLSbtV1DeSXVrJrfyV5pZUUHKgi/0Dk74IDlVRUf/gDv4/BsEH9GZHcn+xhA/nYxDSGDe7H8EH9SR/Uj/RB/UgdmEBKYgJ942LvBM0wQ2EJcK2ZPQLMAsp1PEFEWlPX0EheaSXbSw6zY98hduw7zI59h9m5r5I9B6s/1DYxIY7RQwcwemgip2UNZdSQAYwaMoCMIf0ZmTyAYYP6xeSHfXsFFgpm9jCRBdHTzKwA+CkQD+Du9wJLiSw/mAtUAlcFVYuI9AzVdQ3kFh8it/gQ7xVXNG3v2l9JfeM/Z45TkhIYl5bEmRPTGJuayNjURDJTIn9SkhJCmXbpLYI8++jyozzvwHeCen8R6b4aGp2d+w+zdU8FW/ZUsHXPQd7be4id+w/zwWd/XB9jbGoi2cMGcsHUEUxIH8i49CQmpA0kOTE+3A70Yj3u1tki0rNU1tazefdBNhYdZPPug2wqOsjWvRVU1zUCkTn+rNQkJg0fxKdOHsUJwweRPXwgWalJJPTVNE9XUyiISKeprK1nY9FB1hWUs6GwnHUFZWzfd7jpnPshifFMGTmYL84ay4kjBjF55GAmDhtI//i4cAuXJgoFEemQuoZGtu6p4N38Mtbll7G2oIzc4kNN0z/DB/fjpIxkPn3yKKaOSmbqqMGMTO6v+f5uTqEgIu2yu7yK1bvKeDfvAGvyy9hQVN40BZSSlMDJo5OZP20k0zOSmT46mWGD+4dcsXSEQkFEPsLd2bKngtV5B1i58wArdpZScKAKgIS+fTgpI5kvnD6WGZlDmDlmCKOHDtAIoJdQKIgIjY3O1r0VvLBxD1t2V7B8Zymlh2sBSBvYj9OyhnLlnCymjkrm1LFDdQC4F1MoiMQgd+f9kkO8/f5+3s7dz7Id+zlQWQdAUkIcF0wbwRnjUpmROYTsYQM1CoghCgWRGJG3v5J3duzn7dx9vP3+fooragDIGDKAuZOHM3t8KrPGpzB6aGLIlUqYFAoivVR1XQMrdpby6tYSXttWQm7xISAyHTRnQiqzJ6Ry5oQ0MlMVAvJPCgWRXmTHvsO8sqWYp9cWsWX3QWrqG0no24dZ41L4fM4YZk9IZeqowZoOkjYpFER6sMZGZ31hOS9u2sOLG/fyXnQ0MD4tiU+eNJJPnzyKWeNTSEzQj7q0j75TRHqY2vpGlu3Yz4sb9/K3TXvZc7CauD7GGeNT+OKsTD5x4nBNCUmHKRREeoBDNfW8trWEFzft4eUtxVRU1zMgPo5zTkjn/KnDOfeEYQxJTAi7TOkFFAoi3VRxRTV/31zMixv38FbufmobGklNSuDCaSM5f+pwzpyYpnsGSadTKIh0I3n7K3luw25e3LSX1XkHcIfMlESumDOW86eO4JTMocT10UFiCY5CQSRke8qreWZtEX9dV8TagnIApmUM5vrzJnH+1OGcMHyQzhaSLqNQEAlBWWUtS9fv4ak1hazYWYp7JAhuWnAiF540kjEpOlAs4VAoiHSR6roGXt5SzJOrC3ltWzF1Dc6E9CS+N3cSF80Yxbi0pLBLFFEoiATJ3Vm16wBPrC7kkRV5uEfWGbhidhafmZmhC8mk21EoiASgqKyKx1cV8PiqAvJKKxkQH8dnZmRwwdQRzJsyXAeLpdtSKIh0ktr6Rl7avJeHl+fxZu4+3GH2+FSum5vNgmkjSOqnHzfp/vRdKnKc8vZX8uDyXTyxqoB9h2oZldyf734im0tPHa0DxtLjKBREOqC+oZGXNhfz4LJdvPHePuL6GHNPHMblszL5eHa6poekx1IoiByDvQerWbwinweX5bHnYDUjk/tz/XmT+PxpYxiRrDWJpedTKIgchbuzOq+MP769kyVriwA4KzuNWxdO5RMnDqNvnJamlN5DoSDShpr6Bpau3819b+5kfWE5g/r35aozs/jC6ZlkDx8UdnkigVAoiLRQUlHDQ8vy+POyXZRU1DAhPYnbFk7l4lNG6wwi6fX0HS4SlVtcwR/e2MGTqwupbWjk7EnpfPXScZw1MY0+OnAsMUKhIDFv1a5S/ufV7by0eS/9+vbh0pzRfPVj45iQPjDs0kS6nEJBYlJjo/PylmLufmkbG4sOMjQxnn+dm81XZo8ldWC/sMsTCY1CQWJKfUMjS9YWsej17WzZU0HGkAH82/mT+OrHxmkdYxECDgUzmw/8FxAH/MHdf9Hi+Uzgj8CQaJsb3X1pkDVJbKqpb+Avqwv59d/fo6i8mowhA7jr0pO5aMYo4nVKqUiTwELBzOKAe4B5QAGwwsyWuPumZs1+Aix29/8xsynAUiArqJok9lTVNvDAOzu5782d7DlYzfTRydz86SmcP2WEDh6LtCLIkcLpQK67bwcws0eAhUDzUHBgcHQ7GSgKsB6JIVW1Dfzhje384c0dlFfVMS1jMHdcMp2PZ6fpVtUiRxBkKGQA+c32C4BZLdrcArxoZt8FkoDzWnshM7sauBogMzOz0wuV3qOqtoEHl+3i3tfeZ9+hWs4Yn8I150zk45PSwy5NpEcIMhRa+3XMW+xfDtzv7neZ2WzgATOb5u6NH/oi90XAIoCcnJyWryFCdV0DjyzP464Xt1FRU8/HJqbxvfOyyclKCbs0kR4lyFAoAMY02x/NR6eHvgbMB3D3f5hZfyANKA6wLulF6hoaeXRFPve8ksvu8mpmjUvhurnZnDkxLezSRHqkIENhBZBtZuOAQuAy4Ast2uQBc4H7zWwy0B8oCbAm6SUaG50nVhfwq5feo7CsiskjB3PXpScze0KqjhmIHIfAQsHd683sWuAFIqeb3ufuG83sVmCluy8BbgB+b2bXE5lautLdNT0kR/RW7j5+8dwW1heWMyZlAIu+fCrzpgxXGIh0gkCvU4hec7C0xWM3N9veBJwZZA3Se2zdU8Htz23m1a0lTdcZfGZmhha0EelEuoRTur29B6v51UvbeHRFPkn9+vLvF57IFXOy6Nc3LuzSRHodhYJ0W9V1kWsN/vvV96lraOSKOVlc94lshiYlhF2aSK+lUJBux915dv1ubl+6hcKyKs6fMpwff3IyY1OTwi5NpNdTKEi3srGonJ89s4nlO0qZPHIwd146nTkTdHqpSFdRKEi3UF5Vx3++sJUHl+1iSGIC//EvJ/H508boILJIF1MoSKjcnaXr93DLMxvZf6iGr8zO4vrzJpGcGB92aSIxSaEgoSksq+Lmpzbw9y3FTB01mPuuOI2TRieHXZZITFMoSJdraHTuf3snd724FXf4yScnc+WcLPpqXQOR0CkUpEttKCzn3/+ynnUF5ZxzQjq3LZzGmJTEsMsSkSiFgnSJytp67v7bNu57aydDExP4zeUz+dT0kbo1hUg3o1CQwL2ytZif/GUDhWVVXH76GG6cP1kHkkW6KYWCBKakooZb/7qJZ9YWMXHYQBZ/czanj9P6BiLdmUJBOl1jo7N4ZT7/sXQz1XWNXH/eJL51znjdq0ikB1AoSKfase8wP3piHct3lHL6uBRuv/gkJqQPDLssEWknhYJ0CnfnwWV5/PzZzcTHGXd89iQuPXUMfXRFskiPolCQ41Z8sJofPrGOV7eWcFZ2GndecjIjkvuHXZaIdIBCQY7L8xv2cNOT66isbeBnF03ly2eM1ehApAdTKEiHHK6p59ZnNvHoynymZQzmV5+fycRhOnYg0tMpFOSYrSso418fWcPO/Ye55pwJXD9vEvG6RYVIr6BQkHZzd+57aye/eG4z6QP78dDXz2D2hNSwyxKRTqRQkHYpq6zl3x5bx0ub9zJvynDuvGQ6QxK1LKZIb6NQkKNatesA331oNSWHarj5U1O46sws3bNIpJdSKEibGhudP7y5nV8+v5WRQ/rzxLfnMH30kLDLEpEAKRSkVeWVddzw2Bpe2lzMgmkj+MVnp5M8QDexE+ntFAryERsKy/nWn1ex92A1P/30FK6co+kikVihUJAPeWxlPj95agMpSQks/uZsZmYODbskEelCCgUBoLa+kf/z7Cb+9I9dzJmQyq8vn0nawH5hlyUiXUyhIJRU1PCdB1ezfGcp3zhrHD+af6LWSxaJUQqFGLehsJyr/7SS0spa/uuyGSyckRF2SSISIoVCDHtu/W6uX7yGlMQEHv/WHKZlJIddkoiELNA5AjObb2ZbzSzXzG5so83nzGyTmW00s4eCrEci3J3fvvwe335wNZNHDuapa89UIIgIEOBIwczigHuAeUABsMLMlrj7pmZtsoGbgDPd/YCZDQuqHomoqW/gpifX8+TqQv5lZga3X3wS/eO1TKaIRAQ5fXQ6kOvu2wHM7BFgIbCpWZtvAPe4+wEAdy8OsJ6YV15Zx9UPrGTZjlKuP28S182dqOsPRORDggyFDCC/2X4BMKtFm0kAZvYWEAfc4u7Pt3whM7sauBogMzMzkGJ7u4IDlVz1/1awc/9hHVAWkTYFGQqt/Qrqrbx/NnAOMBp4w8ymuXvZh77IfRGwCCAnJ6fla8hRrC8o56r7V1BT38CfvjpLt7sWkTYFeaC5ABjTbH80UNRKm6fdvc7ddwBbiYSEdJLXtpXw+UX/oF/fPvzlmjkKBBE5oiBDYQWQbWbjzCwBuAxY0qLNU8C5AGaWRmQ6aXuANcWUp9cU8rX7VzA2NYm/XDOHicMGhV2SiHRzgYWCu9cD1wIvAJuBxe6+0cxuNbOLos1eAPab2SbgFeAH7r4/qJpiyQP/2Mm/PrKGU8cO5dFvnsGwwf3DLklEegBz71lT9Dk5Ob5y5cqwy+i2Itcg5HLX37Zx3uRh/PYLp+iUUxHBzFa5e87R2umK5l7E3fn5s5v5w5s7uHhmBndcMp143cNIRI6BQqGXaGx0fvzUBh5enseVc7K4+VNT6NNH1yCIyLFRKPQC9Q2N/PCJdTy5upBrzpnADy44QReliUiHKBR6uPqGRq5fvJZn1hbx/XmTuG6uzugVkY5TKPRgzQPhxgUn8q2zJ4Rdkoj0cAqFHqq+oZHvPbqGv67bzU0LTuSbCgQR6QQ6NaUHamx0fvjEOgWCiHQ6hUIP4+785OkNPLm6kBvmTVIgiEinUij0IO7O7c9t4aFleXz7nAlc+4mJYZckIr3MEUPBzPqY2ZyuKkaO7Lcv57Lo9e18ZfZYfqjTTkUkAEcMBXdvBO7qolrkCP749k7u+ts2Lp6ZwS2fnqpAEJFAtGf66EUz+6zpUyg0T68p5JZnNjJvynB+ecl0XaksIoFpzymp3weSgAYzqyKyeI67++BAKxMgsh7Cvz22ltOzUvjN5TPpq3sZiUiAjhoK7q6b8IdkfUE51/x5FROHDeL3V+TobqciErh2XbxmZhcDHyOynOYb7v5UoFUJefsruer+5QxJTOD+q05jcP/4sEsSkRhw1FAws/8GJgIPRx/6lpnNc/fvBFpZDCs9XMtX7ltGfaPzyFdPZ7gWyBGRLtKekcLZwDSPrsZjZn8E1gdaVQyrrmvga39cQVF5NQ9/YxYThw0MuyQRiSHtOWq5Fchstj8GWBdMObHN3bnhsbWsyS/j15fN4NSxKWGXJCIxpj0jhVRgs5ktj+6fBvzDzJYAuPtFbX6lHJN7X9vOs9H7Gc2fNjLsckQkBrUnFAYAC5rtG3AHcFsgFcWo17eVcOcLW/jU9JFc/fHxYZcjIjGqPaHQ191fa/6AmQ1o+Zh0XH5pJd99+F0mDR/ELy+ZrquVRSQ0bYaCmX0buAYYb2bNjyEMAt4KurBYUVXbwNUPrMLd+d2XTyUxQUtciEh4jvQJ9BDwHHA7cGOzxyvcvTTQqmKEu3PTk+vYsucg9115GmNTk8IuSURiXJuh4O7lQDlwedeVE1v+31s7eWpNETfMm8S5JwwLuxwREa2nEJZ3tu/n50s3c/6U4XznXK2LICLdg0IhBEVlVXznwdVkpSZy1+dO1l1PRaTbUCh0seq6Br7951XU1Dfyuy/nMEj3NBKRbkSnunQhd+enT29kbUE5v/vyqbqFhYh0OxopdKGHlufx6Mp8rj13IhdMHRF2OSIiH6FQ6CIbCsv52ZJNnD0pnevnTQq7HBGRVikUusDhmnque/hdUpISuPvzM4jTgWUR6aYCDQUzm29mW80s18xuPEK7S8zMzSwnyHrC8rNnNrJj/2Hu/vwMUpISwi5HRKRNgYWCmcUB9xC5md4U4HIzm9JKu0HAdcCyoGoJ0/MbdrN4ZQHXnDOB2RNSwy5HROSIghwpnA7kuvt2d68FHgEWttLuNuCXQHWAtYSiuKKam55cz7SMwXzvPB1HEJHuL8hQyADym+0XRB9rYmYzgTHu/tcjvZCZXW1mK81sZUlJSedXGgB354ePr6OqroG7PzeD+DgdvhGR7i/IT6rWjqZ605NmfYC7gRuO9kLuvsjdc9w9Jz09vRNLDM6fl+Xx6tYSblowmezhg8IuR0SkXYIMhQIiS3d+YDRQ1Gx/EDANeNXMdgJnAEt6w8HmXfsPc/vSzZyVncZXZo8NuxwRkXYLMhRWANlmNs7MEoDLgCUfPOnu5e6e5u5Z7p4FvANc5O4rA6wpcI2Nzg8eW0ecGXd8VgvmiEjPElgouHs9cC3wArAZWOzuG83sVjPrtes6P/DOLpbvLOXmT09h1JABYZcjInJMAr33kbsvBZa2eOzmNtqeE2QtXSG/tJI7nt/Cxyelc8mpo8MuR0TkmOmUmE7i7vz7X9ZjwO0Xn6RpIxHpkRQKneTJ1YW88d4+frTgRDI0bSQiPZRCoROUHq7ltmc3cUrmEL40S2cbiUjPpVDoBL94bjOHquv5xWenaxU1EenRFArH6d28AyxeWcBXPzaOSbpITUR6OIXCcWhsdH66ZCPDB/fjurnZYZcjInLcFArH4bFV+awrKOemBZMZ2E8rm4pIz6dQ6KDyqjrueH4rOWOHsnDGqLDLERHpFAqFDvqvl97jQGUtt1w0VdckiEivoVDogO0lh3jgnZ1cdtoYpmUkh12OiEinUSh0wF0vbiM+rg/fn3dC2KWIiHQqhcIxWldQxrPrd/P1s8aTPqhf2OWIiHQqhcIx+uXzW0lJSuAbZ40LuxQRkU6nUDgGb763jzdz9/GdcycyqH982OWIiHQ6hUI7uTt3PL+FjCED+OKszLDLEREJhEKhnZ7bsIf1heV877xs+sfHhV2OiEggFArtUN/QyH++sJXsYQO5+BQtniMivZdCoR0eW1XA9n2H+cEFJxCnu6CKSC+mUDiK6roGfvXSNk7JHMK8KcPDLkdEJFAKhaO497X32Xuwhh/NP1G3sxCRXk+hcARVtQ386R+7OCs7jVnjU8MuR0QkcAqFI3hoeR6lh2u1VoKIxAyFQhvqGhr53ze2c3pWCqdlpYRdjohIl1AotGHJmiKKyqv51jnjwy5FRKTLKBRa4e78/o3tnDB8EOeeMCzsckREuoxCoRWvv7ePLXsq+PpZ43TGkYjEFIVCK+57cwfpg/pxkZbZFJEYo1BoYfPug7y2rYQvzRpLv766x5GIxBaFQguPLM8D4Au6E6qIxCCFQjOHa+p5cnUhn5kxSquqiUhMCjQUzGy+mW01s1wzu7GV579vZpvMbJ2Z/d3MxgZZz9E8s7aIipp6vnRGqGWIiIQmsFAwszjgHmABMAW43MymtGj2LpDj7tOBx4FfBlVPezy4LI8Thg/i1LFDwyxDRCQ0QY4UTgdy3X27u9cCjwALmzdw91fcvTK6+w4Q2mIFGwrLWV9YzhfPyNRpqCISs4IMhQwgv9l+QfSxtnwNeK61J8zsajNbaWYrS0pKOrHEf1q8Mp+Evn1YePKRShQR6d2CDIXWft32VhuafQnIAe5s7Xl3X+TuOe6ek56e3oklRtTUN/D0miIumDqC5MT4Tn99EZGeom+Ar10AjGm2PxooatnIzM4Dfgyc7e41AdbTple2FFNeVcclp2qpTRGJbUGOFFYA2WY2zswSgMuAJc0bmNlM4HfARe5eHGAtR/Tw8nzSBvbjzAlaM0FEYltgoeDu9cC1wAvAZmCxu280s1vN7KJoszuBgcBjZrbGzJa08XKBKa+q4/X3SlgwbQR943TZhojEtiCnj3D3pcDSFo/d3Gz7vCDfvz1e3LgHd/jk9JFhlyIiErqY/9X42fW7GT10ALPGaSEdEZGYDoWD1XW8lbuPBdNG6NoEERFiPBRe2VJMXYMzf9qIsEsREekWYjoUXty0l7SB/Zg5Rre1EBGBGA6F2vpGXt9awtwTh9Gnj6aOREQghkPhtW0lVNTUM3ey1mAWEflAzIbCip2lAJw5MS3kSkREuo+YDYXXt5Uwe3wqSf0CvVRDRKRHiclQ2Heohi17KjhrkkYJIiLNxWQovLBxDwCzx+teRyIizcVkKKzadQCAaRnJIVciItK9xGQobCo6yIwxQ4jXDfBERD4k5j4VD9XUs3VvBWdP6vzFekREerqYC4W1+WW4wyljdRWziEhLMRcKa/LLAJgxekjIlYiIdD8xFwpr88sYn5aktZhFRFoRc6GwobBcZx2JiLQhpkKhrLKWovJqpo4aHHYpIiLdUkyFwrvR4wknjBgUciUiIt1TTIVC4YEqACakDwy5EhGR7immQmF7yWEGxMeRMWRA2KWIiHRLMRUKeaWHyUxJ1KI6IiJtiKlQyC+tYkxKYthliIh0WzETCu5OYVkVo4dq6khEpC0xEwoHq+s5VFPPqCH9wy5FRKTbiplQ2HuwGoARyRopiIi0JWZCYV9FDQDpA/uFXImISPcVM6FQWlkLQEpSQsiViIh0XzETCrX1jQD0j4+ZLouIHLOY+YSsb3QA4nSNgohIm2InFBoioaAlOEVE2hboJ6SZzTezrWaWa2Y3tvJ8PzN7NPr8MjPLCqqW+sbI9JFGCiIibQssFMwsDrgHWABMAS43syktmn0NOODuE4G7gTuCqqdppNBHIwURkbYE+Ql5OpDr7tvdvRZ4BFjYos1C4I/R7ceBuWYWyK/yTSOFOI0URETaEmQoZAD5zfYLoo+12sbd64FyILXlC5nZ1Wa20sxWlpSUdKiYrNQkLjxpBPEKBRGRNvUN8LVb+/T1DrTB3RcBiwBycnI+8nx7nD91BOdPHdGRLxURiRlBjhQKgDHN9kcDRW21MbO+QDJQGmBNIiJyBEGGwgog28zGmVkCcBmwpEWbJcAV0e1LgJfdvUMjAREROX6BTR+5e72ZXQu8AMQB97n7RjO7FVjp7kuA/wUeMLNcIiOEy4KqR0REji7IYwq4+1JgaYvHbm62XQ1cGmQNIiLSfjppX0REmigURESkiUJBRESaKBRERKSJ9bQzQM2sBNjVwS9PA/Z1Yjk9gfocG9Tn2HA8fR7r7ulHa9TjQuF4mNlKd88Ju46upD7HBvU5NnRFnzV9JCIiTRQKIiLSJNZCYVHYBYRAfY4N6nNsCLzPMXVMQUREjizWRgoiInIECgUREWnSK0MqYdYbAAAEiElEQVTBzOab2VYzyzWzG1t5vp+ZPRp9fpmZZXV9lZ2rHX3+vpltMrN1ZvZ3MxsbRp2d6Wh9btbuEjNzM+vxpy+2p89m9rno//VGM3uoq2vsbO343s40s1fM7N3o9/eFYdTZWczsPjMrNrMNbTxvZvbr6L/HOjM7pVMLcPde9YfIbbrfB8YDCcBaYEqLNtcA90a3LwMeDbvuLujzuUBidPvbsdDnaLtBwOvAO0BO2HV3wf9zNvAuMDS6Pyzsurugz4uAb0e3pwA7w677OPv8ceAUYEMbz18IPEdk5cozgGWd+f69caRwOpDr7tvdvRZ4BFjYos1C4I/R7ceBuWbWkxdvPmqf3f0Vd6+M7r5DZCW8nqw9/88AtwG/BKq7sriAtKfP3wDucfcDAO5e3MU1drb29NmBwdHtZD66wmOP4u6vc+QVKBcCf/KId4AhZjays96/N4ZCBpDfbL8g+lirbdy9HigHUrukumC0p8/NfY3Ibxo92VH7bGYzgTHu/teuLCxA7fl/ngRMMrO3zOwdM5vfZdUFoz19vgX4kpkVEFm/5btdU1pojvXn/ZgEushOSFr7jb/lebftadOTtLs/ZvYlIAc4O9CKgnfEPptZH+Bu4MquKqgLtOf/uS+RKaRziIwG3zCzae5eFnBtQWlPny8H7nf3u8xsNpHVHKe5e2Pw5YUi0M+v3jhSKADGNNsfzUeHk01tzKwvkSHnkYZr3V17+oyZnQf8GLjI3Wu6qLagHK3Pg4BpwKtmtpPI3OuSHn6wub3f20+7e5277wC2EgmJnqo9ff4asBjA3f8B9Cdy47jeql0/7x3VG0NhBZBtZuPMLIHIgeQlLdosAa6Ibl8CvOzRIzg91FH7HJ1K+R2RQOjp88xwlD67e7m7p7l7lrtnETmOcpG7rwyn3E7Rnu/tp4icVICZpRGZTtrepVV2rvb0OQ+YC2Bmk4mEQkmXVtm1lgBfiZ6FdAZQ7u67O+vFe930kbvXm9m1wAtEzly4z903mtmtwEp3XwL8L5EhZi6REcJl4VV8/NrZ5zuBgcBj0WPqee5+UWhFH6d29rlXaWefXwDON7NNQAPwA3ffH17Vx6edfb4B+L2ZXU9kGuXKnvxLnpk9TGT6Ly16nOSnQDyAu99L5LjJhUAuUAlc1anv34P/7UREpJP1xukjERHpIIWCiIg0USiIiEgThYKIiDRRKIiISBOFgkgHmNl1ZrbZzB4MuxaRzqRTUkU6wMy2AAuiVw0frW2cuzd0QVkix00jBZFjZGb3ErmV8xIzKzezB8zsZTN7z8y+EW1zTvQe/w8B60MtWOQYaKQg0gHR+ynlANcC/0Lk3kpJRNYymEXk9hLPAtPaM5oQ6S40UhA5fk+7e5W77wNeIbIGAMByBYL0NAoFkePXcrj9wf7hri5E5HgpFESO30Iz629mqURuZLYi5HpEOkyhIHL8lhM5fvAOcJu79+jlICW26UCzyHEws1uAQ+7+n2HXItIZNFIQEZEmGimIiEgTjRRERKSJQkFERJooFEREpIlCQUREmigURESkyf8HSmvHUfFyOhcAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from matplotlib import pyplot as plt \n",
    "plt.title(\"wd_roc\") \n",
    "plt.xlabel(\"fpr\") \n",
    "plt.ylabel(\"tpr\") \n",
    "plt.plot(fpr,tpr) \n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2.预测"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from WD_models/model_WIDE_AND_DEEP_1567750302\\model.ckpt-3689\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    }
   ],
   "source": [
    "test_proba = []\n",
    "for i in range(2556790):\n",
    "    one_res = next(test_results)[\"probabilities\"]\n",
    "    test_proba.append(one_res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_proba = np.array(test_proba)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2556790, 2)"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_proba.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "3.提交结果"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "def submit(submission_file):\n",
    "    f = open(submission_file,\"w+\")\n",
    "    ocolnames = [\"id\",\"target\"]\n",
    "    f.write(\",\".join(ocolnames)+\"\\n\")\n",
    "    outcols = []\n",
    "    for i in range(len(test_proba)):\n",
    "        col1 = i\n",
    "        col2 = test_proba[:,1][i]\n",
    "        outcols = [str(col1),str(col2)]\n",
    "        f.write(\",\".join(outcols)+\"\\n\")\n",
    "    f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "submit(\"./submission/submission_wd.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "submission_wd= pd.read_csv(\"./submission/submission_wd.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2556790, 2)"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "submission_wd.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0.309036</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0.313812</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>0.094208</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>0.068431</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>0.181079</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>0.110071</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>6</td>\n",
       "      <td>0.111275</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>7</td>\n",
       "      <td>0.259496</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>8</td>\n",
       "      <td>0.150790</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>9</td>\n",
       "      <td>0.628534</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>10</td>\n",
       "      <td>0.693512</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>11</td>\n",
       "      <td>0.222664</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>12</td>\n",
       "      <td>0.204592</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>13</td>\n",
       "      <td>0.703435</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>14</td>\n",
       "      <td>0.202987</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>15</td>\n",
       "      <td>0.583714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>16</td>\n",
       "      <td>0.673884</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>17</td>\n",
       "      <td>0.380842</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>18</td>\n",
       "      <td>0.331869</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>19</td>\n",
       "      <td>0.491079</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    id    target\n",
       "0    0  0.309036\n",
       "1    1  0.313812\n",
       "2    2  0.094208\n",
       "3    3  0.068431\n",
       "4    4  0.181079\n",
       "5    5  0.110071\n",
       "6    6  0.111275\n",
       "7    7  0.259496\n",
       "8    8  0.150790\n",
       "9    9  0.628534\n",
       "10  10  0.693512\n",
       "11  11  0.222664\n",
       "12  12  0.204592\n",
       "13  13  0.703435\n",
       "14  14  0.202987\n",
       "15  15  0.583714\n",
       "16  16  0.673884\n",
       "17  17  0.380842\n",
       "18  18  0.331869\n",
       "19  19  0.491079"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "submission_wd.head(20)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
